import re
import requests
from lxml import etree

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
}
url1 = 'http://www.cs.ecitic.com/newsite/cpzx/jrcpxxgs/zgcp/index.html'
res = requests.get(url=url1, headers=headers, verify=False)  # verify关键字参数，在请求的时候不验证网站的ca证书
res.encoding = res.apparent_encoding
# # 大部分情况可以自动解码，实在不行可手动设置编码
# res.encoding = 'utf-8'
total_num = int(re.search('var countPage = (?P<page>\d+)//共多少页', res.text).group('page'))
# var countPage = 103//共多少页

def get_data(url):
    url = url.replace('\n', '')
    print(url)
    response = requests.get(url=url,
                            headers=headers, verify=False)
    response.encoding = response.apparent_encoding
    data = response.text
    tree = etree.HTML(data)
    rows = tree.xpath('/html/body/div[4]/div/ul/li')

    for row in rows:
        row_mc = row.xpath('./span[@class="th1"]/text()')
        row_glr = row.xpath('./span[@class="th2"]/text()')
        row_fxpj = row.xpath('./span[@class="th3"]/text()')
        row_je = row.xpath('./span[@class="th4"]/text()')
        row_gs = row.xpath('./span[@class="th5"]/text()')
        sting = f"{row_mc}, {row_glr}, {row_fxpj}, {row_je}, {row_gs},'\n'"

        sting = sting.replace("['", "").replace("']", "").replace("'", "")
        with open('./zxzq/中信证券.txt', 'a+', encoding='utf-8') as f:
            f.write(sting)


start_page = int(input('请输入需要抓取的页码开始：（1开始）'))

end_page = int(input(f'请输入需要抓取的页码结束：（{total_num}结束）'))

for i in range(start_page - 1, end_page):
    if end_page + 1 > total_num:
        print('页码超过数据限制！！')
        exit(-1)
    if i == 0:
        url = 'http://www.cs.ecitic.com/newsite/cpzx/jrcpxxgs/zgcp/index.html'
        get_data(url)
    else:
        url = f'http://www.cs.ecitic.com/newsite/cpzx/jrcpxxgs/zgcp/index_{i}.html'
        get_data(url)
